GAA EVAL.R
Changhua Yu
4/2/2019
A demo for GAA Submission evaluation package
source("../GAA-EVAL.R")## Warning: package 'ggplot2' was built under R version 3.4.4## Warning: package 'dplyr' was built under R version 3.4.4## Warning: package 'pROC' was built under R version 3.4.4## Warning: package 'reshape2' was built under R version 3.4.3## Warning: package 'ggrepel' was built under R version 3.4.4## Warning: package 'gmodels' was built under R version 3.4.4Read in the Experimental data provided by CAGI
exp.data <- read.RealData(file = "exp_data.csv", sep = ",",
col.id = 2, col.value = 5, col.sd = 6)Read in the submission folders
sub.data <- read.Submission.Folder(folder.name = "prediction/",col.id = 1,
col.value = 2, col.sd = 3, real.data = exp.data)ScatterPlot inspection
plot_all_scatter(real.data = exp.data, pred.data = sub.data, z.transform = TRUE)## Warning: Removed 4 rows containing missing values (geom_point).
## Warning: Removed 4 rows containing missing values (geom_point).
Correlation-based Evaluation
# 1. Render coefficient value
result.cor.pearson <- eval.Correlation(real.data = exp.data, pred.data = sub.data,
method = "pearson", sd.use = 0.3,z.transform = TRUE)
head(result.cor.pearson)## pearson.coefficient.n=355.sd<0.3
## Group_1-prediction_file_nocomments-1 0.3108669
## Group_1-prediction_file_nocomments-2 0.2509373
## Group_1-prediction_file_nocomments-3 0.2175485
## Group_2-prediction_file_nocomments-1 0.2723800
## Group_2-prediction_file_nocomments-2 0.2800299
## Group_3-prediction_file_nocomments-1 0.3149842
## p.value
## Group_1-prediction_file_nocomments-1 2.157713e-09
## Group_1-prediction_file_nocomments-2 1.683052e-06
## Group_1-prediction_file_nocomments-3 3.563631e-05
## Group_2-prediction_file_nocomments-1 1.859902e-07
## Group_2-prediction_file_nocomments-2 8.082396e-08
## Group_3-prediction_file_nocomments-1 1.287089e-09# 2. Plot Correlation
plot.Correlation(result.cor.pearson, "Pearson")
RMSD-based Evaluation
result.rmsd <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = FALSE, density.distance.adjust = FALSE, variance.normalization = FALSE)
head(result.rmsd)## RMSD
## Group_1-prediction_file_nocomments-1 0.4009268
## Group_1-prediction_file_nocomments-2 0.4106208
## Group_1-prediction_file_nocomments-3 0.4289230
## Group_2-prediction_file_nocomments-1 0.4404307
## Group_2-prediction_file_nocomments-2 0.4236929
## Group_3-prediction_file_nocomments-1 0.4624334plot.RMSD(result.rmsd, method="")
result.rmsd <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = FALSE, density.distance.adjust = FALSE, variance.normalization = TRUE)
head(result.rmsd)## RMSD
## Group_1-prediction_file_nocomments-1 2.245247
## Group_1-prediction_file_nocomments-2 2.281298
## Group_1-prediction_file_nocomments-3 2.424075
## Group_2-prediction_file_nocomments-1 2.355892
## Group_2-prediction_file_nocomments-2 2.348380
## Group_3-prediction_file_nocomments-1 1.869974plot.RMSD(result.rmsd, method="")
result.rmsd <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = TRUE, density.distance.adjust = FALSE, variance.normalization = TRUE)
head(result.rmsd)## RMSD
## Group_1-prediction_file_nocomments-1 2.261219
## Group_1-prediction_file_nocomments-2 2.296956
## Group_1-prediction_file_nocomments-3 2.439058
## Group_2-prediction_file_nocomments-1 2.368549
## Group_2-prediction_file_nocomments-2 2.362522
## Group_3-prediction_file_nocomments-1 1.878088plot.RMSD(result.rmsd, method="")
# with variance.normalization
result.rmsd <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = TRUE, density.distance.adjust = FALSE, variance.normalization = FALSE)
head(result.rmsd)## RMSD
## Group_1-prediction_file_nocomments-1 0.4037789
## Group_1-prediction_file_nocomments-2 0.4134391
## Group_1-prediction_file_nocomments-3 0.4315742
## Group_2-prediction_file_nocomments-1 0.4427969
## Group_2-prediction_file_nocomments-2 0.4262445
## Group_3-prediction_file_nocomments-1 0.4644398plot.RMSD(result.rmsd, method="")
result.rmsd <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = TRUE, density.distance.adjust = TRUE, variance.normalization = TRUE)
head(result.rmsd)## RMSD
## Group_1-prediction_file_nocomments-1 2.065760
## Group_1-prediction_file_nocomments-2 2.101465
## Group_1-prediction_file_nocomments-3 2.248217
## Group_2-prediction_file_nocomments-1 2.180030
## Group_2-prediction_file_nocomments-2 2.165433
## Group_3-prediction_file_nocomments-1 1.727107plot.RMSD(result.rmsd, method="")
Cut-off-based Evaluation
result.auc.0.6 <- eval.AUC(real.data =exp.data, pred.data = sub.data,
threshold = 0.6)## [1] "Group_1-prediction_file_nocomments-1"
## [1] "Group_1-prediction_file_nocomments-2"
## [1] "Group_1-prediction_file_nocomments-3"
## [1] "Group_2-prediction_file_nocomments-1"
## [1] "Group_2-prediction_file_nocomments-2"
## [1] "Group_3-prediction_file_nocomments-1"
## [1] "Group_3-prediction_file_nocomments-2"
## [1] "Group_3-prediction_file_nocomments-3"
## [1] "Group_4-prediction_file_nocomments-1"
## [1] "Group_4-prediction_file_nocomments-2"
## [1] "Group_4-prediction_file_nocomments-3"
## [1] "Group_5-prediction_file_nocomments-1"
## [1] "Group_5-prediction_file_nocomments-2"
## [1] "Group_6-prediction_file_nocomments-1"
## [1] "Group_6-prediction_file_nocomments-2"
## [1] "Group_6-prediction_file_nocomments-3"
## [1] "Group_6-prediction_file_nocomments-4"
## [1] "Group_6-prediction_file_nocomments-5"
## [1] "Group_6-prediction_file_nocomments-6"
## [1] "Group_6-prediction_file_nocomments-7"
## [1] "Group_7-prediction_file_nocomments-1"
## [1] "Group_7-prediction_file_nocomments-2"
## [1] "Group_7-prediction_file_nocomments-3"
## [1] "Group_7-prediction_file_nocomments-4"
## [1] "Group_7-prediction_file_nocomments-5"
## [1] "Group_7-prediction_file_nocomments-6"plot.AUC(result.auc.0.6)
Between-method Evaluation
# for all the submission files
result.bM.spearman <-eval.Correlation.Between(real.data = exp.data, pred.data = sub.data,
method = "spearman",sd.use = 0.5,z.transform = TRUE)
plot.Correlation.Between(result.bM.spearman$coefficient, method="Spearman")
# for best submission of each group
result.bM.spearman <-eval.Correlation.Between(real.data = exp.data, pred.data = sub.data,
method = "pearson",sd.use = 0.5,z.transform = TRUE,grouped = TRUE)## Warning: package 'bindrcpp' was built under R version 3.4.4plot.Correlation.Between(result.bM.spearman$coefficient, method="pearson")
Partial-Correlation Evaluation
result.pCor <- eval.Partial.Correlation(real.data = exp.data, pred.data = sub.data, method = "spearman")## Loading required package: ppcor## Loading required package: MASS## Warning: package 'MASS' was built under R version 3.4.4##
## Attaching package: 'MASS'## The following object is masked from 'package:dplyr':
##
## selectplot.Correlation.Between(result.bM.spearman$coefficient, method="Spearman")
PCA Plot
total = cbind(real = exp.data$value,sub.data$value)
Plot.PCA(na.omit(total), labels=F, legend=TRUE) 
Uniqueness Evaluation
# uniqueness as adj.r^2 difference between total linear model and linear models without certain group
result.uniq = eval.uniqueness(real.data = exp.data, pred.data = sub.data)
result.uniq## uniqueness
## 1 -0.0020608244
## 2 -0.0017316046
## 3 0.0021973873
## 4 -0.0004628426
## 5 -0.0025660668
## 6 -0.0008577268
## 7 0.0311478355plot.uniqueness(result.uniq, method="")
Bootstrap Analysis
For Correlation-based Evaluation, provide mean, CI, and median pval
# 1. Render coefficient value
boot.result.cor.pearson <- eval.Correlation(real.data = exp.data, pred.data = sub.data,
method = "pearson", sd.use = 0.3,z.transform = TRUE,boot = T)
head(boot.result.cor.pearson)## avg low_ci high_ci
## Group_1-prediction_file_nocomments-1 0.3022523 0.2211897 0.3790684
## Group_1-prediction_file_nocomments-2 0.2419690 0.1590423 0.3246424
## Group_1-prediction_file_nocomments-3 0.2104707 0.1258205 0.2958989
## Group_2-prediction_file_nocomments-1 0.2646057 0.1791920 0.3389399
## Group_2-prediction_file_nocomments-2 0.2721590 0.1906289 0.3551322
## Group_3-prediction_file_nocomments-1 0.3120721 0.2377822 0.3892085
## sd p.value
## Group_1-prediction_file_nocomments-1 0.04725837 4.847518e-09
## Group_1-prediction_file_nocomments-2 0.04896693 4.137076e-06
## Group_1-prediction_file_nocomments-3 0.05055433 6.100567e-05
## Group_2-prediction_file_nocomments-1 0.04897955 4.193340e-07
## Group_2-prediction_file_nocomments-2 0.04954591 2.180379e-07
## Group_3-prediction_file_nocomments-1 0.04764063 1.459542e-09# 2. Plot Correlation
plot.Correlation(boot.result.cor.pearson, "Pearson",boot = TRUE)
For RMSD-based Evaluation
boot.result.rmsd <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = TRUE, density.distance.adjust = TRUE, variance.normalization = TRUE,boot = TRUE)
head(boot.result.rmsd)## RMSD low_ci high_ci sd
## Group_1-prediction_file_nocomments-1 2.076652 1.882741 2.273068 0.11688112
## Group_1-prediction_file_nocomments-2 2.110861 1.923630 2.300080 0.11113260
## Group_1-prediction_file_nocomments-3 2.258749 2.050444 2.465838 0.12281248
## Group_2-prediction_file_nocomments-1 2.193060 2.008141 2.394741 0.11986462
## Group_2-prediction_file_nocomments-2 2.178519 1.994105 2.363846 0.11710324
## Group_3-prediction_file_nocomments-1 1.733068 1.592175 1.884086 0.09022407plot.RMSD(boot.result.rmsd, method="",boot = TRUE)
boot.result.rmsd2 <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = FALSE, density.distance.adjust = FALSE, variance.normalization = TRUE,boot = TRUE)
head(boot.result.rmsd2)## RMSD low_ci high_ci sd
## Group_1-prediction_file_nocomments-1 2.247001 2.054898 2.443938 0.11643457
## Group_1-prediction_file_nocomments-2 2.282272 2.101136 2.470210 0.11153860
## Group_1-prediction_file_nocomments-3 2.427660 2.231988 2.637035 0.12251575
## Group_2-prediction_file_nocomments-1 2.361926 2.176524 2.560581 0.11631509
## Group_2-prediction_file_nocomments-2 2.354202 2.171926 2.546089 0.11466239
## Group_3-prediction_file_nocomments-1 1.874147 1.726995 2.037892 0.09521376plot.RMSD(boot.result.rmsd2, method="",boot = TRUE)
boot.result.rmsd3 <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = TRUE, density.distance.adjust = TRUE, variance.normalization = FALSE,boot = TRUE)
head(boot.result.rmsd3)## RMSD low_ci high_ci
## Group_1-prediction_file_nocomments-1 0.3686599 0.3451944 0.3914462
## Group_1-prediction_file_nocomments-2 0.3776528 0.3543891 0.3999812
## Group_1-prediction_file_nocomments-3 0.3971755 0.3737324 0.4209869
## Group_2-prediction_file_nocomments-1 0.4069886 0.3798605 0.4314241
## Group_2-prediction_file_nocomments-2 0.3901605 0.3662721 0.4129206
## Group_3-prediction_file_nocomments-1 0.4272323 0.4022508 0.4511513
## sd
## Group_1-prediction_file_nocomments-1 0.01388509
## Group_1-prediction_file_nocomments-2 0.01412916
## Group_1-prediction_file_nocomments-3 0.01448195
## Group_2-prediction_file_nocomments-1 0.01463802
## Group_2-prediction_file_nocomments-2 0.01391602
## Group_3-prediction_file_nocomments-1 0.01559804plot.RMSD(boot.result.rmsd3, method="",boot = TRUE)
boot.result.rmsd4 <- eval.RMSD(real.data = exp.data, pred.data = sub.data,sd.use = NA,
density.distance = FALSE, density.distance.adjust = FALSE, variance.normalization = FALSE,boot = TRUE)
head(boot.result.rmsd4)## RMSD low_ci high_ci
## Group_1-prediction_file_nocomments-1 0.4013498 0.3808202 0.4230032
## Group_1-prediction_file_nocomments-2 0.4104614 0.3899481 0.4311438
## Group_1-prediction_file_nocomments-3 0.4288221 0.4058096 0.4509057
## Group_2-prediction_file_nocomments-1 0.4402073 0.4186930 0.4616345
## Group_2-prediction_file_nocomments-2 0.4234627 0.4017325 0.4446576
## Group_3-prediction_file_nocomments-1 0.4625610 0.4365683 0.4866667
## sd
## Group_1-prediction_file_nocomments-1 0.01273138
## Group_1-prediction_file_nocomments-2 0.01293515
## Group_1-prediction_file_nocomments-3 0.01319231
## Group_2-prediction_file_nocomments-1 0.01295168
## Group_2-prediction_file_nocomments-2 0.01237304
## Group_3-prediction_file_nocomments-1 0.01516553plot.RMSD(boot.result.rmsd4, method="",boot = TRUE)
For Uniqueness Evaluation
result.uniq = eval.uniqueness(real.data = exp.data, pred.data = sub.data,boot = TRUE)
result.uniq## uniqueness low_ci high_ci sd
## 1 -6.124713e-06 -0.002461942 0.006430638 0.003398813
## 2 5.234856e-04 -0.002466881 0.007993327 0.003543444
## 3 4.840517e-03 -0.002351803 0.020059159 0.007771222
## 4 1.565600e-03 -0.002408887 0.010856149 0.004678369
## 5 3.084723e-04 -0.005832817 0.010483546 0.005352871
## 6 1.652259e-03 -0.002422145 0.012441364 0.005363944
## 7 3.205858e-02 0.008929731 0.059251561 0.015497702plot.uniqueness(result.uniq, method="",boot = TRUE)